import type { Metadata } from "next";
import Link from "next/link";
import { notFound } from "next/navigation";
import { ArrowLeft, ChevronDown, RefreshCw } from "lucide-react";
import { getWorkspace } from "@/lib/session";
import { internalApi, InternalApiError, type CrawlJob, type CrawlPage, type CrawlPagesPage } from "@/lib/api";
import { formatBytes, formatDate, formatMs, formatNumber, timeAgo } from "@/lib/format";
import { PageHeader } from "@/components/ui/page-header";
import { Alert } from "@/components/ui/alert";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { CopyButton } from "@/components/ui/copy-button";
import { EmptyState } from "@/components/ui/empty-state";
import { Stat, StatGrid } from "@/components/ui/stat";
import { Table, TableBody, TableCell, TableEmpty, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import { CrawlPageStatusBadge, CrawlStatusBadge } from "@/components/dashboard/crawls/crawl-status-badge";
import { CancelCrawlButton } from "@/components/dashboard/crawls/cancel-crawl-button";
export const dynamic = "force-dynamic";
export const metadata: Metadata = { title: "Crawl" };
const PAGE_SIZE = 100;
const PREVIEW_CHARS = 2000;
const PAGE_STATUSES = ["success", "blocked", "failed"] as const;
type SearchParams = Promise<{ cursor?: string | string[]; status?: string | string[] }>;
function first(v: string | string[] | undefined): string | undefined {
return Array.isArray(v) ? v[0] : v;
}
function Row({ label, children, mono }: { label: string; children: React.ReactNode; mono?: boolean }) {
return (
{label}
{children}
);
}
function optionValue(v: unknown): string {
if (v === undefined || v === null) return "—";
if (Array.isArray(v)) return v.length ? v.join(", ") : "—";
if (typeof v === "object") return JSON.stringify(v);
return String(v);
}
const OPTION_KEYS = ["max_pages", "max_depth", "format", "same_domain", "allow_subdomains", "respect_robots", "use_sitemap", "concurrency", "delay_ms", "timeout", "main_content", "country", "network", "browser", "browser_fallback", "include_patterns", "exclude_patterns", "webhook_url"] as const;
export default async function CrawlDetailPage({ params, searchParams }: { params: Promise<{ id: string }>; searchParams: SearchParams }) {
const [ws, { id }, sp] = await Promise.all([getWorkspace(), params, searchParams]);
if (!/^crawl_[A-Za-z0-9]{4,64}$/.test(id)) notFound();
const cursor = first(sp.cursor)?.trim() || null;
const statusFilter = first(sp.status)?.trim() || null;
const status = statusFilter && (PAGE_STATUSES as readonly string[]).includes(statusFilter) ? statusFilter : null;
let job: CrawlJob | null = null;
let pages: CrawlPagesPage = { data: [], next_cursor: null };
let loadError: string | null = null;
try {
job = await internalApi.getCrawl(ws.project.id, ws.user.id, id);
} catch (e) {
if (e instanceof InternalApiError && (e.status === 404 || e.code === "NOT_FOUND")) notFound();
loadError = e instanceof InternalApiError ? e.message : "The Fetcha API service is unreachable.";
}
if (!job) {
return (
Crawls
{loadError ?? "Unknown error."}
);
}
try {
pages = await internalApi.crawlPages(ws.project.id, ws.user.id, id, { cursor, limit: PAGE_SIZE, status });
} catch (e) {
loadError = e instanceof InternalApiError ? e.message : "Could not load the crawled pages.";
}
const active = job.status === "queued" || job.status === "running";
const stats = job.stats ?? { discovered: 0, fetched: 0, ok: 0, blocked: 0, failed: 0, bytes: 0 };
const hrefFor = (next: { cursor?: string | null; status?: string | null }) => {
const q = new URLSearchParams();
const st = next.status === undefined ? status : next.status;
if (st) q.set("status", st);
if (next.cursor) q.set("cursor", next.cursor);
const s = q.toString();
return `/dashboard/crawls/${job!.id}${s ? `?${s}` : ""}`;
};
// Only shown for finished jobs (a live elapsed counter would need a client component).
const durationMs = job.started_at && job.completed_at ? new Date(job.completed_at).getTime() - new Date(job.started_at).getTime() : null;
return (
Crawls
{job.label ?? job.domain ?? job.seed_url}
}
description={
{job.id}
·
{job.seed_url}
·
created {timeAgo(job.created_at)}
}
actions={
<>
{active ? (
) : null}
{active ? : null}
>
}
/>
{job.error ? (
{job.error.message}
) : null}
{active ? (
{job.status === "queued" ? "The job is waiting for a worker slot. " : "Pages are being fetched. "}
This page does not update on its own; use Refresh to see progress.
) : null}
Pages
{loadError ? (
{loadError}
) : null}
URL
Status
HTTP
Depth
Bytes
Duration
Mode
{pages.data.length === 0 ? (
{active ? "No pages fetched yet." : status ? `No ${status} pages.` : "No pages were fetched."}
) : (
pages.data.map((p) => )
)}
{formatNumber(pages.data.length)} page{pages.data.length === 1 ? "" : "s"} shown{cursor ? " (continued)" : ""}
{cursor ? (
) : null}
{pages.next_cursor ? (
) : null}
{!pages.data.length && !active && !loadError && !status ? : null}
);
}
function FilterLink({ href, active, children }: { href: string; active: boolean; children: React.ReactNode }) {
return (
{children}
);
}
function PageRow({ page: p }: { page: CrawlPage }) {
const preview = typeof p.content === "string" && p.content.length ? p.content.slice(0, PREVIEW_CHARS) : null;
const truncated = typeof p.content === "string" && p.content.length > PREVIEW_CHARS;
return (
{p.url}
{p.title ? (
{p.title}
) : null}
{p.error_code ? {p.error_code} : null}
{p.http_status ?? —}
{p.depth}
{formatBytes(p.bytes)}
{formatMs(p.duration_ms)}
{p.mode ? {p.mode === "browser" ? "Browser" : "HTTP"} : —}
{p.final_url && p.final_url !== p.url ? (
<>
- Final URL
- {p.final_url}
>
) : null}
{p.description ? (
<>
- Description
- {p.description}
>
) : null}
- Content type
- {p.content_type ?? "—"}
- Links
- {p.links_count ?? "—"}
- Fetched
- {p.fetched_at ? formatDate(p.fetched_at, { timeStyle: "medium" }) : "—"}
{preview ? (
<>
{preview}
{truncated ?
Showing the first {formatNumber(PREVIEW_CHARS)} of {formatNumber(p.content!.length)} characters.
: null}
>
) : (
No stored content for this page.
)}
);
}